{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "14676b65-69ca-409b-860a-87eb7649d8f8",
   "metadata": {},
   "source": [
    "# 导包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8ba72f79-e2da-4ab8-8c63-435c0bd701f6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\Administrator\\Envs\\tf-gpu\\lib\\site-packages\\keras\\src\\losses.py:2976: The name tf.losses.sparse_softmax_cross_entropy is deprecated. Please use tf.compat.v1.losses.sparse_softmax_cross_entropy instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.losses import sparse_categorical_crossentropy\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow.keras.optimizers import Adam"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28457132-601a-4a88-bbfc-b13274bb507e",
   "metadata": {},
   "source": [
    "# 数据加载"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9a558d2b-701e-4270-95ab-4f96aab53909",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "      <th>len_text</th>\n",
       "      <th>max_idx_word</th>\n",
       "      <th>min_idx_word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>[2967, 6758, 339, 2021, 1854, 3731, 4109, 3792...</td>\n",
       "      <td>1057</td>\n",
       "      <td>7539</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11</td>\n",
       "      <td>[4464, 486, 6352, 5619, 2465, 4802, 1452, 3137...</td>\n",
       "      <td>486</td>\n",
       "      <td>7543</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>[7346, 4068, 5074, 3747, 5681, 6093, 1777, 222...</td>\n",
       "      <td>764</td>\n",
       "      <td>7543</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>[7159, 948, 4866, 2109, 5520, 2490, 211, 3956,...</td>\n",
       "      <td>1570</td>\n",
       "      <td>7543</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>[3646, 3055, 3055, 2490, 4659, 6065, 3370, 581...</td>\n",
       "      <td>307</td>\n",
       "      <td>7539</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label                                               text  len_text  \\\n",
       "0      2  [2967, 6758, 339, 2021, 1854, 3731, 4109, 3792...      1057   \n",
       "1     11  [4464, 486, 6352, 5619, 2465, 4802, 1452, 3137...       486   \n",
       "2      3  [7346, 4068, 5074, 3747, 5681, 6093, 1777, 222...       764   \n",
       "3      2  [7159, 948, 4866, 2109, 5520, 2490, 211, 3956,...      1570   \n",
       "4      3  [3646, 3055, 3055, 2490, 4659, 6065, 3370, 581...       307   \n",
       "\n",
       "   max_idx_word  min_idx_word  \n",
       "0          7539            19  \n",
       "1          7543            23  \n",
       "2          7543            25  \n",
       "3          7543            19  \n",
       "4          7539            13  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('./data/pre_train.cav')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec626635-e03f-434c-af08-0f7886d37a2f",
   "metadata": {},
   "source": [
    "- max_word_idx = 7549"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1b2c2d3e-26f3-472b-9c02-54b158ab6c17",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>len_text</th>\n",
       "      <th>max_idx_word</th>\n",
       "      <th>min_idx_word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "      <td>200000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.210950</td>\n",
       "      <td>907.207110</td>\n",
       "      <td>7531.504910</td>\n",
       "      <td>41.329170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>3.084955</td>\n",
       "      <td>996.029036</td>\n",
       "      <td>50.793614</td>\n",
       "      <td>65.703618</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2465.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>374.000000</td>\n",
       "      <td>7539.000000</td>\n",
       "      <td>19.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>676.000000</td>\n",
       "      <td>7543.000000</td>\n",
       "      <td>23.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>5.000000</td>\n",
       "      <td>1131.000000</td>\n",
       "      <td>7543.000000</td>\n",
       "      <td>25.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>13.000000</td>\n",
       "      <td>57921.000000</td>\n",
       "      <td>7549.000000</td>\n",
       "      <td>2539.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               label       len_text   max_idx_word   min_idx_word\n",
       "count  200000.000000  200000.000000  200000.000000  200000.000000\n",
       "mean        3.210950     907.207110    7531.504910      41.329170\n",
       "std         3.084955     996.029036      50.793614      65.703618\n",
       "min         0.000000       2.000000    2465.000000       0.000000\n",
       "25%         1.000000     374.000000    7539.000000      19.000000\n",
       "50%         2.000000     676.000000    7543.000000      23.000000\n",
       "75%         5.000000    1131.000000    7543.000000      25.000000\n",
       "max        13.000000   57921.000000    7549.000000    2539.000000"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "54558935-e23f-4133-afc9-5b41fb2b0cd0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 200000 entries, 0 to 199999\n",
      "Data columns (total 5 columns):\n",
      " #   Column        Non-Null Count   Dtype \n",
      "---  ------        --------------   ----- \n",
      " 0   label         200000 non-null  int64 \n",
      " 1   text          200000 non-null  object\n",
      " 2   len_text      200000 non-null  int64 \n",
      " 3   max_idx_word  200000 non-null  int64 \n",
      " 4   min_idx_word  200000 non-null  int64 \n",
      "dtypes: int64(4), object(1)\n",
      "memory usage: 7.6+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "28dc52b4-1157-4e90-ab06-bd86d54fc85e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "label\n",
       "0     38918\n",
       "1     36945\n",
       "2     31425\n",
       "3     22133\n",
       "4     15016\n",
       "5     12232\n",
       "6      9985\n",
       "7      8841\n",
       "8      7847\n",
       "9      5878\n",
       "10     4920\n",
       "11     3131\n",
       "12     1821\n",
       "13      908\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a5a26a63-d39f-4e66-9f1f-6db70cbd00c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[:10000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a1b3ac75-a6ac-4c36-bff4-77d718128d09",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "str"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(df['text'].values[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "105e7a71-1365-4106-9429-f0b2389196d4",
   "metadata": {},
   "source": [
    "# 建模"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9b9543a4-6523-402a-bae1-68170b9eea0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 计算类别权重\n",
    "from sklearn.utils.class_weight import compute_class_weight\n",
    "\n",
    "class_weights = compute_class_weight('balanced',classes=np.unique(df['label']),y=df['label'])\n",
    "class_weight_dict = dict(enumerate(class_weights))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ed644da3-357a-40ec-aad4-edf890cdf5ed",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 0.36707215904502505,\n",
       " 1: 0.38667517352048414,\n",
       " 2: 0.45459711330833047,\n",
       " 3: 0.6454486190626795,\n",
       " 4: 0.9513661618083568,\n",
       " 5: 1.1678968513500887,\n",
       " 6: 1.4307175048286715,\n",
       " 7: 1.6158482395333431,\n",
       " 8: 1.8205319594385478,\n",
       " 9: 2.43036990229913,\n",
       " 10: 2.9036004645760745,\n",
       " 11: 4.56266824839166,\n",
       " 12: 7.844983133286264,\n",
       " 13: 15.733165512901195}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_weight_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "93e39f5b-456a-4927-9bad-43b137d3ac77",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_data(df, max_len=512):\n",
    "    # 转换文本格式\n",
    "    texts = [eval(t) if isinstance(t, str) else t for t in df['text']]\n",
    "    \n",
    "    # 序列填充/截断\n",
    "    X = pad_sequences(texts, maxlen=max_len, padding='post', truncating='post')\n",
    "    \n",
    "    # 处理标签\n",
    "    y = tf.keras.utils.to_categorical(df['label'], num_classes=14)\n",
    "    \n",
    "    # 分层分割\n",
    "    X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, stratify=df['label'],random_state=42)\n",
    "    return X_train, X_val, y_train, y_val\n",
    "X_train, X_val, y_train, y_val = preprocess_data(df, max_len=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a07b4148-c444-4a1c-a730-23700d1368b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(vocab_size=7550, max_len=512, num_classes=14):\n",
    "    inputs = layers.Input(shape=(max_len,))\n",
    "    \n",
    "    # 嵌入层（不再使用mask_zero，因为后续卷积会破坏掩码）\n",
    "    embedding = layers.Embedding(input_dim=vocab_size, output_dim=128)(inputs)\n",
    "    \n",
    "    # 双向LSTM分支\n",
    "    lstm = layers.Bidirectional(layers.LSTM(128, return_sequences=True))(embedding)\n",
    "    \n",
    "    # 卷积分支（改用普通Conv1D）\n",
    "    conv = layers.Conv1D(filters=64, kernel_size=3, activation='relu')(embedding)\n",
    "    conv = layers.GlobalMaxPooling1D()(conv)\n",
    "    conv = layers.RepeatVector(max_len)(conv)  # 对齐序列长度\n",
    "    \n",
    "    # 注意力机制（输入维度对齐）\n",
    "    attention = layers.Attention()([lstm, lstm])\n",
    "    pool = layers.GlobalAveragePooling1D()(attention)\n",
    "    \n",
    "    # 特征融合\n",
    "    merged = layers.concatenate([pool, layers.Flatten()(conv)])\n",
    "    \n",
    "    # 分类头\n",
    "    outputs = layers.Dense(num_classes, activation='softmax',\n",
    "                         kernel_regularizer=tf.keras.regularizers.l2(0.01))(merged)\n",
    "    \n",
    "    return tf.keras.Model(inputs=inputs, outputs=outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1ca259c9-704d-4794-997a-4e3805454ff6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\Administrator\\Envs\\tf-gpu\\lib\\site-packages\\keras\\src\\backend.py:1398: The name tf.executing_eagerly_outside_functions is deprecated. Please use tf.compat.v1.executing_eagerly_outside_functions instead.\n",
      "\n",
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                Output Shape                 Param #   Connected to                  \n",
      "==================================================================================================\n",
      " input_1 (InputLayer)        [(None, 512)]                0         []                            \n",
      "                                                                                                  \n",
      " embedding (Embedding)       (None, 512, 128)             966400    ['input_1[0][0]']             \n",
      "                                                                                                  \n",
      " conv1d (Conv1D)             (None, 510, 64)              24640     ['embedding[0][0]']           \n",
      "                                                                                                  \n",
      " bidirectional (Bidirection  (None, 512, 256)             263168    ['embedding[0][0]']           \n",
      " al)                                                                                              \n",
      "                                                                                                  \n",
      " global_max_pooling1d (Glob  (None, 64)                   0         ['conv1d[0][0]']              \n",
      " alMaxPooling1D)                                                                                  \n",
      "                                                                                                  \n",
      " attention (Attention)       (None, 512, 256)             0         ['bidirectional[0][0]',       \n",
      "                                                                     'bidirectional[0][0]']       \n",
      "                                                                                                  \n",
      " repeat_vector (RepeatVecto  (None, 512, 64)              0         ['global_max_pooling1d[0][0]']\n",
      " r)                                                                                               \n",
      "                                                                                                  \n",
      " global_average_pooling1d (  (None, 256)                  0         ['attention[0][0]']           \n",
      " GlobalAveragePooling1D)                                                                          \n",
      "                                                                                                  \n",
      " flatten (Flatten)           (None, 32768)                0         ['repeat_vector[0][0]']       \n",
      "                                                                                                  \n",
      " concatenate (Concatenate)   (None, 33024)                0         ['global_average_pooling1d[0][\n",
      "                                                                    0]',                          \n",
      "                                                                     'flatten[0][0]']             \n",
      "                                                                                                  \n",
      " dense (Dense)               (None, 14)                   462350    ['concatenate[0][0]']         \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 1716558 (6.55 MB)\n",
      "Trainable params: 1716558 (6.55 MB)\n",
      "Non-trainable params: 0 (0.00 Byte)\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = build_model()\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6eea6585-1fe4-40da-861c-2800a98a64d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "WARNING:tensorflow:From C:\\Users\\Administrator\\Envs\\tf-gpu\\lib\\site-packages\\keras\\src\\utils\\tf_utils.py:492: The name tf.ragged.RaggedTensorValue is deprecated. Please use tf.compat.v1.ragged.RaggedTensorValue instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\Administrator\\Envs\\tf-gpu\\lib\\site-packages\\keras\\src\\engine\\base_layer_utils.py:384: The name tf.executing_eagerly_outside_functions is deprecated. Please use tf.compat.v1.executing_eagerly_outside_functions instead.\n",
      "\n",
      "250/250 [==============================] - 122s 478ms/step - loss: 1.9410 - accuracy: 0.4812 - auc: 0.8745 - precision: 0.8515 - recall: 0.3069 - val_loss: 1.0605 - val_accuracy: 0.7640 - val_auc: 0.9671 - val_precision: 0.8710 - val_recall: 0.6820 - lr: 3.0000e-04\n",
      "Epoch 2/5\n",
      "250/250 [==============================] - 121s 484ms/step - loss: 0.7178 - accuracy: 0.8179 - auc: 0.9847 - precision: 0.8827 - recall: 0.7554 - val_loss: 0.8052 - val_accuracy: 0.8145 - val_auc: 0.9815 - val_precision: 0.8852 - val_recall: 0.7520 - lr: 3.0000e-04\n",
      "Epoch 3/5\n",
      "250/250 [==============================] - 118s 470ms/step - loss: 0.4577 - accuracy: 0.8867 - auc: 0.9934 - precision: 0.9265 - recall: 0.8512 - val_loss: 0.6735 - val_accuracy: 0.8470 - val_auc: 0.9853 - val_precision: 0.8952 - val_recall: 0.7990 - lr: 3.0000e-04\n",
      "Epoch 4/5\n",
      "250/250 [==============================] - 118s 473ms/step - loss: 0.3288 - accuracy: 0.9273 - auc: 0.9970 - precision: 0.9507 - recall: 0.8974 - val_loss: 0.6212 - val_accuracy: 0.8550 - val_auc: 0.9862 - val_precision: 0.8990 - val_recall: 0.8145 - lr: 3.0000e-04\n",
      "Epoch 5/5\n",
      "250/250 [==============================] - 119s 475ms/step - loss: 0.2518 - accuracy: 0.9539 - auc: 0.9985 - precision: 0.9679 - recall: 0.9349 - val_loss: 0.5638 - val_accuracy: 0.8720 - val_auc: 0.9865 - val_precision: 0.9125 - val_recall: 0.8400 - lr: 3.0000e-04\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "model.compile(\n",
    "    optimizer=Adam(learning_rate=3e-4),\n",
    "    loss='categorical_crossentropy',\n",
    "    metrics=['accuracy', \n",
    "             tf.keras.metrics.AUC(name='auc'),\n",
    "             tf.keras.metrics.Precision(name='precision'),\n",
    "             tf.keras.metrics.Recall(name='recall')]\n",
    ")\n",
    "\n",
    "callbacks = [\n",
    "    EarlyStopping(patience=3, monitor='val_auc', mode='max'),\n",
    "    tf.keras.callbacks.ReduceLROnPlateau(factor=0.5, patience=2),\n",
    "    # tf.keras.callbacks.TensorBoard(log_dir='./logs/')\n",
    "]\n",
    "\n",
    "history = model.fit(\n",
    "    X_train, y_train,\n",
    "    validation_data=(X_val, y_val),\n",
    "    batch_size=32,\n",
    "    epochs=5,\n",
    "    class_weight=class_weight_dict,\n",
    "    callbacks=callbacks\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a90bb1c4-299e-4846-ae39-64e75f3b9053",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('all_model.keras')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4070d98c-49f0-4c60-acad-fd1c1efb0365",
   "metadata": {},
   "source": [
    "|                | 预测为正类 (Positive) | 预测为负类 (Negative) |\n",
    "|----------------|----------------------|----------------------|\n",
    "| 真实正类 (True)  | True Positive (TP)    | False Negative (FN)   |\n",
    "| 真实负类 (False) | False Positive (FP)   | True Negative (TN)    |"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9d36ac7-9d83-4498-b866-ef6d15898cdb",
   "metadata": {},
   "source": [
    "一、基础指标：判断整体正确性\n",
    "1. Accuracy（准确率）\n",
    "公式：正确预测样本 / 总样本\n",
    "\n",
    "核心意义：模型整体的预测正确率\n",
    "\n",
    "适用场景：类别均衡的二分类/多分类\n",
    "\n",
    "缺陷：\\\n",
    "▶ 当正负样本比例9:1时，全预测为负准确率可达90% \\\n",
    "▶ 对不平衡数据严重失真\n",
    "\n",
    "2. Error Rate（错误率）\n",
    "公式：1 - Accuracy\n",
    "\n",
    "意义：快速判断模型基本错误比例\n",
    "\n",
    "二、细分指标：分析错误类型\n",
    "1. Precision（精确率）\n",
    "公式：TP / (TP + FP)\n",
    "\n",
    "核心意义：模型预测为正的样本中，有多少是真的正例\n",
    "\n",
    "业务影响：减少误报（False Positive）\n",
    "\n",
    "典型场景： \\\n",
    "▶ 垃圾邮件分类（误判正常邮件代价高） \\\n",
    "▶ 金融风控（避免误封正常账户） \n",
    "\n",
    "2. Recall（召回率）\n",
    "公式：TP / (TP + FN)\n",
    "\n",
    "核心意义：真实正例中，模型找出多少\n",
    "\n",
    "业务影响：减少漏检（False Negative）\n",
    "\n",
    "典型场景：\\\n",
    "▶ 疾病诊断（宁可误诊也不漏诊） \\\n",
    "▶ 安全监测（不能放过任何威胁） \n",
    "\n",
    "3. F1-Score\n",
    "公式：2(PrecisionRecall)/(Precision+Recall)\n",
    "\n",
    "核心意义：Precision和Recall的调和平均数\n",
    "\n",
    "适用场景： \\\n",
    "▶ 需要平衡误报和漏报时 \\\n",
    "▶ 类别不平衡且FP/FN代价相近 \n",
    "\n",
    "三、综合指标：评估模型整体能力\n",
    "1. AUC-ROC\n",
    "核心意义：模型对正负样本的区分能力（与阈值无关）\n",
    "\n",
    "取值范围：0.5（随机猜测）~1（完美分类）\n",
    "\n",
    "优势： \\\n",
    "▶ 不受类别分布影响 \\\n",
    "▶ 直观反映排序质量  \n",
    "\n",
    "适用场景： \\\n",
    "▶ 需要比较不同模型的整体性能 \\\n",
    "▶ 分类阈值不确定时 \n",
    "\n",
    "2. PR-AUC\n",
    "核心意义：在正样本稀少时，比ROC更敏感\n",
    "\n",
    "典型场景：\\\n",
    "▶ 极不平衡数据（如欺诈检测）\\\n",
    "▶ 更关注正样本的检测情况 \n",
    "\n",
    "四、业务场景选择指南\n",
    "业务需求\t优先指标\t案例说明\n",
    "减少误报\tPrecision\t垃圾邮件分类（避免误判重要邮件）\n",
    "减少漏检\tRecall\t癌症筛查（宁可误诊也不漏诊）\n",
    "平衡两者\tF1-Score\t客户流失预测（挽留成本≈误判成本）\n",
    "评估整体区分能力\tAUC-ROC\t信用评分模型开发阶段\n",
    "正样本极少\tPR-AUC + Recall\t网络入侵检测（攻击样本<1%）\n",
    "五、记忆技巧\n",
    "Precision → \"预测的正例有多少真\"（查准）\\\n",
    "▶ 联想：狼来了故事中，小孩喊\"狼来了\"的可信度 \n",
    "\n",
    "Recall → \"真实的正例找出多少\"（查全）\\\n",
    "▶ 联想：警察抓小偷，100个小偷抓到80个→召回率80% \n",
    "\n",
    "F1 → \"跷跷板平衡点\"\\\n",
    "▶ 左手Precision，右手Recall，F1是中间的支点 \n",
    "\n",
    "AUC → \"鉴宝专家能力\"\\\n",
    "▶ 0.5=抛硬币鉴定，1.0=火眼金睛 \n",
    "\n",
    "六、常见误区\n",
    "盲目追求高Accuracy：\\\n",
    "▶ 不平衡数据中可能掩盖问题 \\\n",
    "▶ 解决方案：同时看Confusion Matrix \n",
    "\n",
    "忽视业务代价：\\\n",
    "▶ 医疗场景：Recall比Precision更重要 \\\n",
    "▶ 推荐系统：Precision影响用户体验 \n",
    "\n",
    "过早优化指标：\\\n",
    "▶ 应先确保模型学到有效特征，再针对性优化指标 \n",
    "\n",
    "掌握这些指标的本质差异，就能根据业务需求选择最合适的评估方式！ 🎯"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ca88142-2068-430a-aa2a-2aa8f7718719",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84c51af3-df98-47a3-ad08-6e0a7a5ce9d4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf-gpu",
   "language": "python",
   "name": "tf-gpu"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
